/**
 * @projectName interview_-assistant
 * @package org.example.util
 * @className org.example.util.PageFetcher
 * @copyright Copyright 2025 Thunisoft, Inc All rights reserved.
 */
package org.example.util;

import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.example.pojo.entity.Article;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

import static org.jsoup.nodes.Document.OutputSettings.Syntax.html;

/**
 * PageFetcher
 * @description
 * @author admin
 * @date 2025/4/14 10:39
 * @version TODO
 */
@Component
public class PageFetcher {

    @Value("${crawler.base.url:https://www.nowcoder.com}")
    private String baseUrl;
    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
    public String buildSearchUrl(String keyword, int page) {
        // URL构建逻辑
        String base = "https://www.nowcoder.com/search/all?query=%s&type=all&searchType=面经导航栏&subType=0";
        return page > 1 ? String.format(base + "&page=%d", keyword, page) : String.format(base, keyword);
    }

    public String fetch(String url) throws IOException {
        // HTTP请求逻辑
        try (CloseableHttpClient client = HttpClients.createDefault()) {
            HttpGet request = new HttpGet(url);
            request.setHeader("User-Agent", USER_AGENT);
            request.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            return EntityUtils.toString(client.execute(request).getEntity(), StandardCharsets.UTF_8);
        }
    }
}